{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/miniconda3/envs/yuh/lib/python3.10/site-packages/IPython/core/magics/osm.py:417: UserWarning: This is now an optional IPython functionality, setting dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: robotics\n",
      "Fail...\tPrediction: computation and language\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: cryptography and security\tGround truth: multiagent systems\n",
      "Fail...\tPrediction: multimedia\tGround truth: human-computer interaction\n",
      "Fail...\tPrediction: computational engineering, finance, and science\tGround truth: robotics\n",
      "Fail...\tPrediction: social and information networks\tGround truth: human-computer interaction\n",
      "Fail...\tPrediction: cryptography and security\tGround truth: distributed, parallel, and cluster computing\n",
      "Fail...\tPrediction: mathematical optimization\tGround truth: robotics\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: graphics\n",
      "Fail...\tPrediction: software engineering\tGround truth: computation and language\n",
      "Fail...\tPrediction: hardware architecture\tGround truth: performance\n",
      "Fail...\tPrediction: neural and evolutionary computing\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: information theory\tGround truth: networking and internet architecture\n",
      "Fail...\tPrediction: robotics\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: information theory\n",
      "Fail...\tPrediction: networking and internet architecture\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: human-computer interaction\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: computer science and game theory\n",
      "Fail...\tPrediction: social and information networks\tGround truth: computers and society\n",
      "Fail...\tPrediction: computation and language\tGround truth: sound\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: computational engineering, finance, and science\n",
      "Fail...\tPrediction: neural and evolutionary computing\tGround truth: human-computer interaction\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: graphics\n",
      "Fail...\tPrediction: computation and language\tGround truth: social and information networks\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: information theory\n",
      "Fail...\tPrediction: social and information networks\tGround truth: computer science and game theory\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: databases\n",
      "Fail...\tPrediction: computation and language\tGround truth: software engineering\n",
      "Fail...\tPrediction: computational complexity\tGround truth: formal languages and automata theory\n",
      "Fail...\tPrediction: robotics\tGround truth: information theory\n",
      "Fail...\tPrediction: graphics\tGround truth: computational engineering, finance, and science\n",
      "Fail...\tPrediction: robotics\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: information retrieval\tGround truth: networking and internet architecture\n",
      "Fail...\tPrediction: robotics\tGround truth: other computer science\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: software engineering\n",
      "Fail...\tPrediction: data structures and algorithms\tGround truth: databases\n",
      "Fail...\tPrediction: logic in computer science\tGround truth: discrete mathematics\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: networking and internet architecture\n",
      "Fail...\tPrediction: data structures and algorithms\tGround truth: computational complexity\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: hardware architecture\n",
      "Fail...\tPrediction: networking and internet architecture\tGround truth: distributed, parallel, and cluster computing\n",
      "Fail...\tPrediction: computation and language\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: graphics\tGround truth: computational geometry\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: multimedia\n",
      "Fail...\tPrediction: computers and society\tGround truth: human-computer interaction\n",
      "Fail...\tPrediction: computation and language\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: graphics\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: robotics\n",
      "Fail...\tPrediction: software engineering\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: cryptography and security\tGround truth: information theory\n",
      "Fail...\tPrediction: networking and internet architecture\tGround truth: distributed, parallel, and cluster computing\n",
      "Fail...\tPrediction: computers and society\tGround truth: social and information networks\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computational engineering, finance, and science\n",
      "Fail...\tPrediction: computers and society\tGround truth: human-computer interaction\n",
      "Fail...\tPrediction: computation and language\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: computation and language\tGround truth: information theory\n",
      "Fail...\tPrediction: software engineering\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: neural and evolutionary computing\n",
      "Fail...\tPrediction: information retrieval\tGround truth: computers and society\n",
      "Fail...\tPrediction: databases\tGround truth: digital libraries\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: sound\tGround truth: computation and language\n",
      "Fail...\tPrediction: information theory\tGround truth: multimedia\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: computers and society\n",
      "Fail...\tPrediction: software engineering\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: human-computer interaction\n",
      "Fail...\tPrediction: computation and language\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: robotics\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: social and information networks\n",
      "Fail...\tPrediction: computer science and game theory\tGround truth: computers and society\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: operating systems\n",
      "Fail...\tPrediction: computational geometry\tGround truth: data structures and algorithms\n",
      "Fail...\tPrediction: social and information networks\tGround truth: computational complexity\n",
      "Fail...\tPrediction: computation and language\tGround truth: information retrieval\n",
      "Fail...\tPrediction: software engineering\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: multimedia\tGround truth: sound\n",
      "Fail...\tPrediction: cryptography and security\tGround truth: information theory\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: performance\n",
      "Fail...\tPrediction: information retrieval\tGround truth: digital libraries\n",
      "Fail...\tPrediction: cryptography and security\tGround truth: computers and society\n",
      "Fail...\tPrediction: robotics\tGround truth: graphics\n",
      "Fail...\tPrediction: computation and language\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: numerical analysis\tGround truth: mathematical software\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: robotics\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: databases\tGround truth: distributed, parallel, and cluster computing\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: robotics\n",
      "Fail...\tPrediction: computation and language\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: programming languages\tGround truth: software engineering\n",
      "Fail...\tPrediction: cryptography and security\tGround truth: social and information networks\n",
      "Fail...\tPrediction: robotics\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: networking and internet architecture\tGround truth: hardware architecture\n",
      "Fail...\tPrediction: software engineering\tGround truth: computation and language\n",
      "Accuracy: 81.00%\n"
     ]
    }
   ],
   "source": [
    "%cd /hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning/\n",
    "import torch\n",
    "from pathlib import Path\n",
    "import json\n",
    "import re\n",
    "\n",
    "res_dict = {}\n",
    "\n",
    "pred_files = Path(\"checkpoints/higpt-stage2-llama3-sc-epoch30-8192-full_finetune/lightning_logs/version_0/predict\").rglob(\"*.txt\")\n",
    "for pred_file in pred_files:\n",
    "    res_dict[pred_file.stem.strip(\"[]\")] = {\n",
    "        \"prediction\": pred_file.read_text().strip()\n",
    "    }\n",
    "\n",
    "test_data = torch.load(\"dataset/Arxiv2023_instruction_stage2/test_data_list_8192_llama3.pt\")\n",
    "for entry in test_data:\n",
    "    assert str(entry['id']) in res_dict\n",
    "    res_dict[str(entry['id'])][\"ground_truth\"] = entry[\"ground_truth\"]\n",
    "\n",
    "ans_pattern = r\"```(.*?)```\"\n",
    "short_pattern = r\"\\(.*?\\)\"\n",
    "\n",
    "correct_count = 0\n",
    "for res in res_dict.values():\n",
    "    pred = re.findall(ans_pattern, res[\"prediction\"])[0].lower()\n",
    "    truth = re.findall(ans_pattern, res[\"ground_truth\"])[0].lower()\n",
    "    pred_short = re.findall(short_pattern, pred)[0].strip(\"()\").lower()\n",
    "    truth_short = re.findall(short_pattern, truth)[0].strip(\"()\").lower()\n",
    "    pred_full = pred.replace(pred_short, \"\").replace(\"()\", \"\").strip()\n",
    "    truth_full = truth.replace(truth_short, \"\").replace(\"()\", \"\").strip()\n",
    "    if pred_full == truth_full or pred_short == truth_short:\n",
    "        correct_count += 1\n",
    "    else:\n",
    "        print(f\"Fail...\\tPrediction: {pred_full}\\tGround truth: {truth_full}\")\n",
    "\n",
    "print(f\"Accuracy: {correct_count / len(res_dict) * 100:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/miniconda3/envs/yuh/lib/python3.10/site-packages/IPython/core/magics/osm.py:417: UserWarning: This is now an optional IPython functionality, setting dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning\n",
      "Accuracy: 68.60%\n"
     ]
    }
   ],
   "source": [
    "%cd /hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning/\n",
    "import torch\n",
    "from pathlib import Path\n",
    "import json\n",
    "import re\n",
    "\n",
    "res_dict = {}\n",
    "\n",
    "pred_files = Path(\"checkpoints/higpt-stage2-llama3-pr-epoch30-8192-full_finetune/lightning_logs/version_1/predict\").rglob(\"*.txt\")\n",
    "for pred_file in pred_files:\n",
    "    res_dict[pred_file.stem.strip(\"[]\").strip(\"'\")] = {\n",
    "        \"prediction\": pred_file.read_text().strip()\n",
    "    }\n",
    "\n",
    "test_data = torch.load(\"dataset/ICLR_peer_review_stage2/test_data_list_8192_llama3.pt\")\n",
    "for entry in test_data:\n",
    "    assert str(entry['id']) in res_dict\n",
    "    res_dict[str(entry['id'])][\"ground_truth\"] = entry[\"ground_truth\"]\n",
    "\n",
    "ans_pattern = r\"```(.*?)```\"\n",
    "short_pattern = r\"\\(.*?\\)\"\n",
    "\n",
    "correct_count = 0\n",
    "for res in res_dict.values():\n",
    "    pred = re.findall(ans_pattern, res[\"prediction\"])[0].lower()\n",
    "    truth = re.findall(ans_pattern, res[\"ground_truth\"])[0].lower()\n",
    "\n",
    "    if pred == truth:\n",
    "        correct_count += 1\n",
    "    else:\n",
    "        pass\n",
    "        # print(f\"Fail...\\tPrediction: {pred_full}\\tGround truth: {truth_full}\")\n",
    "\n",
    "print(f\"Accuracy: {correct_count / len(res_dict) * 100:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Fail...\tPrediction: reject\tGround truth: accept\n",
      "Fail...\tPrediction: accept\tGround truth: reject\n",
      "Accuracy: 43.40%\n"
     ]
    }
   ],
   "source": [
    "%cd /hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning/\n",
    "import torch\n",
    "from pathlib import Path\n",
    "import json\n",
    "import re\n",
    "\n",
    "res_dict = {}\n",
    "\n",
    "preds = json.load(open(\"inference/ICLR_peer_review_stage2/llama3-70b/res.json\"))\n",
    "\n",
    "for pred in preds:\n",
    "    res_dict[list(pred.keys())[0]] = {\n",
    "        \"prediction\": list(pred.values())[0]\n",
    "    }\n",
    "\n",
    "test_data = torch.load(\"dataset/ICLR_peer_review_stage2/test_data_list_8192_llama3.pt\")\n",
    "for entry in test_data:\n",
    "    assert str(entry['id']) in res_dict\n",
    "    res_dict[str(entry['id'])][\"ground_truth\"] = entry[\"ground_truth\"]\n",
    "\n",
    "ans_pattern = r\"```(.*?)```\"\n",
    "\n",
    "correct_count = 0\n",
    "for res in res_dict.values():\n",
    "    pred = re.findall(ans_pattern, res[\"prediction\"], re.DOTALL)[0].lower().strip()\n",
    "    truth = re.findall(ans_pattern, res[\"ground_truth\"], re.DOTALL)[0].lower().strip()\n",
    "\n",
    "    if pred == truth:\n",
    "        correct_count += 1\n",
    "    else:\n",
    "        pass\n",
    "        print(f\"Fail...\\tPrediction: {pred}\\tGround truth: {truth}\")\n",
    "\n",
    "print(f\"Accuracy: {correct_count / len(res_dict) * 100:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/miniconda3/envs/yuh/lib/python3.10/site-packages/IPython/core/magics/osm.py:417: UserWarning: This is now an optional IPython functionality, setting dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning\n",
      "Fail...\tPrediction: artificial intelligence\tGround truth: robotics\n",
      "Fail...\tPrediction: machine learning\tGround truth: human-computer interaction\n",
      "Fail...\tPrediction: machine learning\tGround truth: neural and evolutionary computing\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: sound\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: databases\n",
      "Fail...\tPrediction: computing research repository\tGround truth: software engineering\n",
      "Fail...\tPrediction: computer science and architecture\tGround truth: operating systems\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: software engineering\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computers and society\n",
      "Fail...\tPrediction: machine learning\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: computational learning\tGround truth: networking and internet architecture\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: information theory\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: graphics\n",
      "Fail...\tPrediction: machine learning\tGround truth: information retrieval\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: social and information networks\n",
      "Fail...\tPrediction: machine learning\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: graphics\n",
      "Fail...\tPrediction: machine learning\tGround truth: information theory\n",
      "Fail...\tPrediction: computing research repository\tGround truth: human-computer interaction\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: robotics\n",
      "Fail...\tPrediction: computational complexity\tGround truth: information retrieval\n",
      "Fail...\tPrediction: artificial intelligence\tGround truth: robotics\n",
      "Fail...\tPrediction: computing research repository\tGround truth: networking and internet architecture\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: multimedia\n",
      "Fail...\tPrediction: computer architecture\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: machine learning\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: machine learning\tGround truth: social and information networks\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: sound\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and robotics\tGround truth: graphics\n",
      "Fail...\tPrediction: computational complexity\tGround truth: software engineering\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computers and society\n",
      "Fail...\tPrediction: machine learning\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: computational complexity\tGround truth: discrete mathematics\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: machine learning\tGround truth: social and information networks\n",
      "Fail...\tPrediction: computer science and artificial intelligence\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer networks\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: sound\n",
      "Fail...\tPrediction: machine learning\tGround truth: sound\n",
      "Fail...\tPrediction: machine learning\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: software engineering\n",
      "Fail...\tPrediction: computing research repository\tGround truth: robotics\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: machine learning\tGround truth: symbolic computation\n",
      "Fail...\tPrediction: computer vision and machine learning\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer science > networking and internet architecture\tGround truth: information theory\n",
      "Fail...\tPrediction: computer vision and robotics\tGround truth: human-computer interaction\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: software engineering\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: multimedia\n",
      "Fail...\tPrediction: computer science and game theory\tGround truth: computational complexity\n",
      "Fail...\tPrediction: computer vision and robotics\tGround truth: robotics\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: machine learning\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: computer vision and robotics\tGround truth: information theory\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computers and society\n",
      "Fail...\tPrediction: machine learning\tGround truth: multiagent systems\n",
      "Fail...\tPrediction: computational biology and neural systems\tGround truth: computational engineering, finance, and science\n",
      "Fail...\tPrediction: machine learning\tGround truth: robotics\n",
      "Fail...\tPrediction: computer science - artificial intelligence\tGround truth: computation and language\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: robotics\n",
      "Fail...\tPrediction: computer science and information theory\tGround truth: databases\n",
      "Fail...\tPrediction: machine learning\tGround truth: social and information networks\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: robotics\n",
      "Fail...\tPrediction: computer vision and robotics\tGround truth: software engineering\n",
      "Fail...\tPrediction: computer science > artificial intelligence > machine learning\tGround truth: robotics\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and robotics\tGround truth: social and information networks\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: digital libraries\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computers and society\n",
      "Fail...\tPrediction: computer science > artificial intelligence\tGround truth: computation and language\n",
      "Fail...\tPrediction: computing in biology and medicine\tGround truth: computers and society\n",
      "Fail...\tPrediction: computing research repository\tGround truth: programming languages\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer architecture\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: machine learning\tGround truth: information retrieval\n",
      "Fail...\tPrediction: computational learning theory\tGround truth: computation and language\n",
      "Fail...\tPrediction: machine learning\tGround truth: information retrieval\n",
      "Fail...\tPrediction: artificial intelligence\tGround truth: logic in computer science\n",
      "Fail...\tPrediction: computer architecture\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: software engineering\n",
      "Fail...\tPrediction: optimization and control\tGround truth: robotics\n",
      "Fail...\tPrediction: computer architecture\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computation and language\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer science and theory of computation\tGround truth: data structures and algorithms\n",
      "Fail...\tPrediction: cryptology and security\tGround truth: information theory\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computers and society\n",
      "Fail...\tPrediction: artificial intelligence\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: graphics\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: computer science and machine learning\tGround truth: software engineering\n",
      "Fail...\tPrediction: computer architecture\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: computational complexity\tGround truth: data structures and algorithms\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computation and language\n",
      "Fail...\tPrediction: computational complexity\tGround truth: formal languages and automata theory\n",
      "Fail...\tPrediction: software engineering\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: computer science > networking and internet architecture\tGround truth: information theory\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: machine learning\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computational geometry\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer science and machine learning\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: computer science and artificial intelligence\tGround truth: computation and language\n",
      "Fail...\tPrediction: computational physics\tGround truth: mathematical software\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer science and machine learning\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: hardware architecture\n",
      "Fail...\tPrediction: computer science and machine learning\tGround truth: social and information networks\n",
      "Fail...\tPrediction: machine learning\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: graphics\n",
      "Fail...\tPrediction: computational complexity\tGround truth: data structures and algorithms\n",
      "Fail...\tPrediction: computational complexity\tGround truth: neural and evolutionary computing\n",
      "Fail...\tPrediction: computing and sustainability\tGround truth: distributed, parallel, and cluster computing\n",
      "Fail...\tPrediction: computer science and machine learning\tGround truth: software engineering\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: sound\n",
      "Fail...\tPrediction: computer science and artificial intelligence\tGround truth: computational complexity\n",
      "Fail...\tPrediction: machine learning\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: software engineering\n",
      "Fail...\tPrediction: computer science and artificial intelligence\tGround truth: information retrieval\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: robotics\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer science and robotics\tGround truth: other computer science\n",
      "Fail...\tPrediction: computational complexity\tGround truth: data structures and algorithms\n",
      "Fail...\tPrediction: computer science and artificial intelligence\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer science > artificial intelligence\tGround truth: robotics\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computational engineering, finance, and science\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computation and language\n",
      "Fail...\tPrediction: neural and evolutionary computing\tGround truth: robotics\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: databases\n",
      "Fail...\tPrediction: machine learning\tGround truth: robotics\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computers and society\n",
      "Fail...\tPrediction: artificial intelligence\tGround truth: information theory\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer science and machine learning\tGround truth: software engineering\n",
      "Fail...\tPrediction: machine learning\tGround truth: information theory\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: performance\n",
      "Fail...\tPrediction: artificial intelligence\tGround truth: robotics\n",
      "Fail...\tPrediction: computer science and game theory\tGround truth: information theory\n",
      "Fail...\tPrediction: computing research repository\tGround truth: distributed, parallel, and cluster computing\n",
      "Fail...\tPrediction: computer vision and robotics\tGround truth: information retrieval\n",
      "Fail...\tPrediction: machine learning\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: machine learning\tGround truth: networking and internet architecture\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computation and language\n",
      "Fail...\tPrediction: machine learning\tGround truth: robotics\n",
      "Fail...\tPrediction: computing research repository\tGround truth: distributed, parallel, and cluster computing\n",
      "Fail...\tPrediction: computer science and game theory\tGround truth: robotics\n",
      "Fail...\tPrediction: computer science and formal languages\tGround truth: software engineering\n",
      "Fail...\tPrediction: computer vision and robotics\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: multimedia\n",
      "Fail...\tPrediction: machine learning\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer science and machine learning\tGround truth: information retrieval\n",
      "Fail...\tPrediction: computing research repository\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computation and language\n",
      "Fail...\tPrediction: computational social science\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer science and machine learning\tGround truth: information retrieval\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: logic in computer science\n",
      "Fail...\tPrediction: computer science and machine learning\tGround truth: software engineering\n",
      "Fail...\tPrediction: machine learning\tGround truth: information retrieval\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computation and language\n",
      "Fail...\tPrediction: software engineering\tGround truth: computation and language\n",
      "Fail...\tPrediction: computational learning theory\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: machine learning\tGround truth: computer vision and pattern recognition\n",
      "Fail...\tPrediction: computer vision and robotics\tGround truth: networking and internet architecture\n",
      "Fail...\tPrediction: machine learning\tGround truth: information retrieval\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computers and society\n",
      "Fail...\tPrediction: machine learning\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: machine learning\tGround truth: sound\n",
      "Fail...\tPrediction: machine learning\tGround truth: computers and society\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: robotics\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: sound\n",
      "Fail...\tPrediction: computer science and machine learning\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: information retrieval\tGround truth: digital libraries\n",
      "Fail...\tPrediction: quantum physics\tGround truth: information theory\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computational geometry\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: graphics\n",
      "Fail...\tPrediction: machine learning\tGround truth: robotics\n",
      "Fail...\tPrediction: distributed, parallel, and cluster computing\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: computer science - artificial intelligence\tGround truth: computation and language\n",
      "Fail...\tPrediction: computer science and artificial intelligence in computer science\tGround truth: information theory\n",
      "Fail...\tPrediction: computer science and game theory\tGround truth: performance\n",
      "Fail...\tPrediction: human-computer interaction\tGround truth: computers and society\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: cryptography and security\n",
      "Fail...\tPrediction: computing research repository\tGround truth: multiagent systems\n",
      "Fail...\tPrediction: computer vision and pattern recognition\tGround truth: computational engineering, finance, and science\n",
      "Fail...\tPrediction: machine learning\tGround truth: robotics\n",
      "Accuracy: 59.40%\n"
     ]
    }
   ],
   "source": [
    "%cd /hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning/\n",
    "import torch\n",
    "from pathlib import Path\n",
    "import json\n",
    "import re\n",
    "\n",
    "res_dict = {}\n",
    "\n",
    "preds = json.load(open(\"inference/Arxiv2023_instruction_stage2/llama3-70b/res.json\"))\n",
    "\n",
    "for pred in preds:\n",
    "    res_dict[list(pred.keys())[0]] = {\n",
    "        \"prediction\": list(pred.values())[0]\n",
    "    }\n",
    "\n",
    "test_data = torch.load(\"dataset/Arxiv2023_instruction_stage2/test_data_list_8192_llama3.pt\")\n",
    "for entry in test_data:\n",
    "    assert str(entry['id']) in res_dict\n",
    "    res_dict[str(entry['id'])][\"ground_truth\"] = entry[\"ground_truth\"]\n",
    "\n",
    "ans_pattern = r\"```(.*?)```\"\n",
    "short_pattern = r\"\\(.*?\\)\"\n",
    "\n",
    "correct_count = 0\n",
    "for res in res_dict.values():\n",
    "    pred = re.findall(ans_pattern, res[\"prediction\"], re.DOTALL)[0].lower()\n",
    "    truth = re.findall(ans_pattern, res[\"ground_truth\"], re.DOTALL)[0].lower()\n",
    "    pred_short = re.findall(short_pattern, pred)[0].strip(\"()\").lower()\n",
    "    truth_short = re.findall(short_pattern, truth)[0].strip(\"()\").lower()\n",
    "    pred_full = pred.replace(pred_short, \"\").replace(\"()\", \"\").strip()\n",
    "    truth_full = truth.replace(truth_short, \"\").replace(\"()\", \"\").strip()\n",
    "    if pred_full == truth_full or pred_short == truth_short:\n",
    "        correct_count += 1\n",
    "    else:\n",
    "        print(f\"Fail...\\tPrediction: {pred_full}\\tGround truth: {truth_full}\")\n",
    "\n",
    "print(f\"Accuracy: {correct_count / len(res_dict) * 100:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning\n",
      "Accuracy: 63.60%\n"
     ]
    }
   ],
   "source": [
    "%cd /hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning/\n",
    "import torch\n",
    "from pathlib import Path\n",
    "import json\n",
    "import re\n",
    "\n",
    "res_dict = {}\n",
    "\n",
    "pred_files = Path(\"checkpoints/higpt-stage2-llama3-MIX-epoch30-8192-full_finetune/lightning_logs/version_2/predict_eval_pr_20240802_16\").rglob(\"*.txt\")\n",
    "for pred_file in pred_files:\n",
    "    res_dict[pred_file.stem.strip(\"[]\").strip(\"'\")] = {\n",
    "        \"prediction\": pred_file.read_text().strip()\n",
    "    }\n",
    "\n",
    "test_data = torch.load(\"dataset/ICLR_peer_review_stage2/test_data_list_8192_llama3.pt\")\n",
    "for entry in test_data:\n",
    "    assert str(entry['id']) in res_dict\n",
    "    res_dict[str(entry['id'])][\"ground_truth\"] = entry[\"ground_truth\"]\n",
    "\n",
    "ans_pattern = r\"```(.*?)```\"\n",
    "short_pattern = r\"\\(.*?\\)\"\n",
    "\n",
    "correct_count = 0\n",
    "for res in res_dict.values():\n",
    "    pred = re.findall(ans_pattern, res[\"prediction\"])[0].lower()\n",
    "    truth = re.findall(ans_pattern, res[\"ground_truth\"])[0].lower()\n",
    "\n",
    "    if pred == truth:\n",
    "        correct_count += 1\n",
    "    else:\n",
    "        pass\n",
    "        # print(f\"Fail...\\tPrediction: {pred_full}\\tGround truth: {truth_full}\")\n",
    "\n",
    "print(f\"Accuracy: {correct_count / len(res_dict) * 100:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/miniconda3/envs/yuh/lib/python3.10/site-packages/IPython/core/magics/osm.py:417: UserWarning: This is now an optional IPython functionality, setting dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning\n",
      "Accuracy: 9.20%\n"
     ]
    }
   ],
   "source": [
    "%cd /hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning/\n",
    "import torch\n",
    "from pathlib import Path\n",
    "import json\n",
    "import re\n",
    "\n",
    "res_dict = {}\n",
    "\n",
    "pred_files = Path(\"checkpoints/vanilla-llama3-pr-epoch30-8192-full_finetune/lightning_logs/version_1/predict\").rglob(\"*.txt\")\n",
    "for pred_file in pred_files:\n",
    "    res_dict[pred_file.stem.strip(\"[]\").strip(\"'\")] = {\n",
    "        \"prediction\": pred_file.read_text().strip()\n",
    "    }\n",
    "\n",
    "test_data = torch.load(\"dataset/ICLR_peer_review_stage2/test_data_list_8192_llama3.pt\")\n",
    "for entry in test_data:\n",
    "    assert str(entry['id']) in res_dict\n",
    "    res_dict[str(entry['id'])][\"ground_truth\"] = entry[\"ground_truth\"]\n",
    "\n",
    "ans_pattern = r\"```(.*?)```\"\n",
    "short_pattern = r\"\\(.*?\\)\"\n",
    "\n",
    "correct_count = 0\n",
    "for res in res_dict.values():\n",
    "    pred = re.findall(ans_pattern, res[\"prediction\"])[0].lower()\n",
    "    truth = re.findall(ans_pattern, res[\"ground_truth\"])[0].lower()\n",
    "\n",
    "    if pred == truth:\n",
    "        correct_count += 1\n",
    "    else:\n",
    "        pass\n",
    "        # print(f\"Fail...\\tPrediction: {pred_full}\\tGround truth: {truth_full}\")\n",
    "\n",
    "print(f\"Accuracy: {correct_count / len(res_dict) * 100:.2f}%\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.14 ('yuh')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "92619f85c62eb73280c07ca2268c8e47b90999a589aa097a1a08a504bd2fb2c6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
